# encoding: UTF-8
'''
Created: 2014-05-28 16:32

@author: VAN
'''
import Queue
import json
import logging
from threading import Thread
import threading
from time import sleep
import time
import urllib2

from orm.models import RingWeb


logger = logging.getLogger('apps')


class ParserThread(Thread):
    def __init__(self, url, cid, startIndex=0, endIndex=0):
        Thread.__init__(self)
        self.urlf = url
        self.cid = cid
        self.startIndex = startIndex
        self.cursorIndex = startIndex
        self.endIndex = endIndex
    
    def run(self):
        while 1:
            if self.endIndex != 0 and self.cursorIndex > self.endIndex:
                break;
            url = self.urlf.format(cid=str(self.cid), page=str(self.cursorIndex))
            
            try:
                req = urllib2.Request(url=url)
                html = urllib2.urlopen(req).read()
                json_str = html[:-2] + html[-1:]
                ja = json.loads(json_str)
                logger.info('url : ' + str(url) + " ### js: " + str(ja))
                hasmore = 0
                filterIndex = False
                for jo in ja:
                    if not filterIndex and jo.get('hasmore'):
                        hasmore = jo.get('hasmore')
                        self.cursorIndex = jo.get('curpage')
                        filterIndex = True
                        print "cursorIndex:" , self.cursorIndex
                    else:
                        oid = jo.get('id')
                        name = jo.get('name')
                        artist = jo.get('artist')
                        duration = jo.get('duration')
                        playcnt = jo.get('playcnt')
                        
                        o = RingWeb()
                        o.id = oid
                        o.name = name
                        o.artist = artist
                        o.duration = duration
                        o.playcnt = playcnt
                        
                        try:
                            o.save()
                        except Exception:
                            logger.info('exception : ' + str(jo))
                
                if hasmore == 0:
                    break
            except Exception:
                logger.info('url error : ' + str(url))
                self.cursorIndex += 1
            
        logger.info(str(self.urlf) + str(self.cid) + ' has finished')


def run():
    print 'worker init success'
#     for cid in [0, 1, 2, 3, 4, 5, 6, 7, 8, 22, 23]:
#         url = 'http://www.shoujiduoduo.com/ringweb/ringweb.php?type=getlist&listid={cid}&page={page}'
#         print url
#         t = ParserThread(url, cid)
#         t.setDaemon(True)
#         t.start()
    url = 'http://www.shoujiduoduo.com/ringweb/ringweb.php?type=getlist&listid={cid}&page={page}'
#     t = ParserThread(url, 2, 698)
#     t.setDaemon(True)
#     t.start()
    
    t1 = ParserThread(url, 22, 1408)
    t1.setDaemon(True)
    t1.start()
    
    print 'worker finish'
    

class ThreadPoolExecutor:
    def __init__(self, coreSize, maxSize, keepAlive, queue):
        self.coreSize = coreSize
        self.maxSize = maxSize
        self.keepAlive = keepAlive
        self.queue = queue
        
        self.workers = []
    
    def add_job(self, func, *args, **kwargs):
            workers = len(self.workers) 
            if workers < self.coreSize:
                self._new_worker()
            elif self.queue.full() and workers < self.maxSize:
                self._new_worker()
            
            self.queue.put((func, args, kwargs))
            
    
    def _new_worker(self):
        worker = ThreadPoolExecutor.Worker(self.queue)
        self.workers.append(worker)
            
    
    class Worker(Thread):
        def __init__(self, queue, *args, **kwargs):
            Thread.__init__(self, *args, **kwargs)
            self.queue = queue
            
            self.timestamp = time.time()
            self.setDaemon(True)
            self.start()
            
        def run(self):
            while 1:
                job_detail = None
                try:
                    (func, args, kwargs) = self.queue.get()
                    job_detail = (func, args, kwargs)
                    func(args, kwargs)
                    self.timestamp = time.time()
                except Queue.Empty:
                    workers = len(self.workers)
                    if time.time() - self.timestamp > self.keepAlive and  workers > self.coreSize:
                        break
                    else:
                        sleep(5)
                except: 
                    logger.info('worker run except' + str(job_detail))
                    

def init():                   
    queue = Queue.Queue(maxsize=1000)
    executor = ThreadPoolExecutor(10, 30, 60, queue)
    objs = RingWeb.objects.exclude(status__in=['1', '2'])[:3000]
    for obj in objs:
        executor.add_job(obj.download)
    # obj = RingWeb.objects.get(id='100000')
     
    # executor.add_job(obj.download())
t = threading.Thread(target=init)
t.setDaemon(True)
t.start()
    
print 'end'
                    
                    

            
            
        
    


        

     
